#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# A command-line-interface parser.
#
# Copyright(C) Ping Yeh 2012.

"""A builder for command line interface.

CliParser eases the task of specifying a command line interface and provides
a simple interface for the results of parsing the command line.

Take some of the options of the bison command as an example:
    parser = (CliParser().Take('--file-prefix').Alias('-b').WithDefault('')
        .Will('Specify a prefix to use for all bison output file names.')
        .Take('-c').WithDefault(False)
        .Take('--defines').As(str)
        .Will('Write  an  extra  output  file containing macro definitions ...')
        .Take('-q').WithDefault(False).Take('--graph').As(str)
        .Take('--debug').Alias('-g').WithDefault(False)
        .Take('--verbose').Alias('-v').As(bool))
    config = parser.Parse(sys.argv)
    # Options with default values are always defined.
    if config.help:  # Use dot notation to access options.
        # Show help and exit
    # Use map notation for options with dashes, slashes, etc.
    # The map notation is available for all options.
    file_prefix = config['file-prefix']  
    # Options without defaults needs a has_key() check before reading.
    if config.has_key('defines'):
        # Do something with config['defines']
    # Loop over arguments.
    for input_file in config:
        # Do something with the input file.
    # Individual arguments can be addressed like a list.
    if len(config) > 2:
        output_file = config[1]

A CLI with subcommands can be built by feeding a CliParser to another like
    foo_parser = CliParser().Take('-x').As(float)
    bar_parser = CliParser().Take('-n').As(int)
    parser = CliParser().Take('foo').Command(foo_parser).Take('bar').Command(bar_parser)
The final parser will accept commands like
   $ example.py foo -x 3.5
   $ example.py bar -n 5
"""

import logging
import os
import re
import sys
import threading

class CalledBeforeTakeError(Exception):
    pass

class OptionNotValidError(Exception):
    pass

class CyclicUseError(Exception):
    pass

# Multiplicity specifications
ONE_OR_MORE = '+'
ZERO_OR_MORE = '*'
ZERO_OR_ONE = '?'

class _CliOption(object):
    """A command line option.
    A CLI option has 3 mandatory attributes: name, date type, doc.
    A CLI option can have 2 optional attributes: alias(es), default value.
    """

    def __init__(self, cli_token):
        self._name = ''
        self._data_type = None
        self.doc = ''
        self._tokens = [cli_token]
        self._default_value = None
        self._action = []
        self._variable_name = ''
        self._multiplicity = 1

    def SetName(self, name):
        if str != type(name):
            raise TypeError('name must be a string')
        self._name = name

    def GetName(self):
        return self._name

    def GetCliTokens(self):
        """Gets a tuple of all CLI tokens of this option."""
        return tuple(self._tokens)

    def AddAlias(self, alias):
        self._tokens.append(alias)

    def GetAction(self):
        return self._action

    def AddAction(self, action):
        logging.debug('adding action %s to option %s', action, self)
        def ConvertToUnicode(data):
            if type(data) == str:
                return unicode(data, 'utf-8')
            elif type(data) == unicode:
                return data
            else:  # should never happen if the data is from command line.
                raise TypeError('Can not convert %s of type %s to unicode' % (
                    data, type(data)))
        if callable(action):
            if action == unicode:
                self._action.append(ConvertToUnicode)
            else:
                self._action.append(action)

    def GetValueType(self):
        return self._data_type

    def SetValueType(self, value_type):
        logging.debug('SetValueType to %s for %s', value_type, self)
        if not type(value_type) is type and not callable(value_type):
            raise TypeError('%s can not be used as a type' % (value_type))
        if (None != self._default_value and
            type(self._default_value) != value_type):
            raise TypeError('%s conflicts with default value %s' % (
                value_type, self._default_value))
        self._data_type = value_type
        self.AddAction(self._data_type)

    def GetDefaultValue(self):
        return self._default_value

    def SetDefaultValue(self, default_value):
        new_type = type(default_value)
        # Check type conflict
        if None != self._data_type and new_type != self._data_type:
            raise TypeError('Default value %s is not of type %s' % (
                default_value, self._data_type))
        self._default_value = default_value
        if None == self._data_type:
            self._data_type = type(default_value)
            self.AddAction(self._data_type)

    def AssertValid(self):
        """Asserts that the option is valid.
        Right now it checks that mandatory attributes are set.
        Returns: None.
        Raises: OptionNotValidError with a message indicating which attribute
            that's making the option erroneous.
        """
        error = None
        if None == self._data_type:
            error = 'Missing data type, use As() to set it.'
        if None != error:
            raise OptionNotValidError('option %s is invalid: %s' % (
                self.GetCliTokens()[0], error))

    def __repr__(self):
        return '(%s %s=%s "%s")' % (self._data_type.__name__, self._tokens[0],
                                    self._default_value, self.doc)

class _CliConfig(object):
    """The result of a CliParser's parsing work."""

    def __init__(self):
        self._arguments = []
        self._options = {}

    def AddArgument(self, arg):
        self._arguments.append(arg)

    def SetOption(self, name, value):
        self._options[name] = value

    def Merge(self, other):
        """Merges with another _CliConfig object."""
        self._arguments.extend(other._arguments)
        self._options.update(other._options)

    def __getitem__(self, name):
        if str == type(name):
            return self._options[name]
        elif int == type(name):
            return self._arguments[name]
        else:
            raise KeyError()

    def __len__(self):
        return len(self._arguments)

    def __getattr__(self, name):
        return self[name]

    def __iter__(self):
        return iter(self._arguments)

    def __repr__(self):
        result = []
        for key, value in self._options.iteritems():
            result.append('%s=%s' % (key, value))
        for a in self._arguments:
            result.append(str(a))
        return '\n'.join(result)


class _AlignmentOption(object):
    both = 'both'
    short = 'short'

class _PlainDecorator(object):
    def DecorateName(self, name):
        return name
    def GetDecoratorLength(self, string):
        return 0

class _AnsiTerminalDecorator(object):
    CSI = '\x1b['
    CSI_regex = '\x1b\['
    def DecorateName(self, name):
        return self.CSI + '4m' + name + self.CSI + '0m'
    def GetDecoratorLength(self, string):
        num_CSI = len(re.findall(self.CSI_regex, string))
        return 4 * num_CSI
        
class _PlainComponentMaker(object):
    def __init__(self, decorator=None):
        self._decorator = decorator
        if None == self._decorator:
            self._decorator = _PlainDecorator()

    def MakeHeader(self, parser):
        output = []
        output.append('Usage: %s [options] ' % parser._program_name)
        if [] != parser._named_arguments:
            for arg in parser._named_arguments:
                output.append('%s ' % arg.GetCliTokens()[0])
        else:
            output.append('[arguments]')
        if parser._description:
            output.append('\n\n%s' % parser._description)
        output.append('\n\nOptions:\n')
        return output

    def _Categorize(self, tokens, short_prefix, long_prefix):
        """Categorize the list of tokens into short and long tokens."""
        short_tokens, long_tokens = [], []
        for token in tokens:
            if token.startswith(long_prefix):
                long_tokens.append(token)
            elif token.startswith(short_prefix):
                short_tokens.append(token)
        return short_tokens, long_tokens

    def MakeLabel(self, option, parser, align):
        """Makes the display label for the option.
        Args:
            option: the CliOption object to work on.
            parser: the CliParser object for auxiliary info.
            align: the alignment setting of CLI tokens.
        Returns:
            a 2-tuple: [0] is the max of raw label lengths of this option.
                       [1] is a list of formatted labels, one element per alias.
        """
        raw_length, formatted = [], []
        name = option.GetName()
        decorated_name = self._decorator.DecorateName(name)
        if _AlignmentOption.both == align:
            value_type = option.GetValueType()
            for token in option.GetCliTokens():  # primary tokens first
                label = token
                if (token.startswith(parser._long_option_prefix) and
                    bool != value_type):
                    label += parser._long_option_delimiter
                elif token.startswith(parser._short_option_prefix):
                    label += ' '
                if value_type != bool:
                    label += decorated_name
                raw_length.append(len(token) + 1 + len(name))
                formatted.append(label)
        elif _AlignmentOption.short == align:
            short_tokens, long_tokens = self._Categorize(
                option.GetCliTokens(), parser._short_option_prefix,
                parser._long_option_prefix)
            full_label = '%2s %s' % (' '.join(short_tokens), 
                                     ' '.join(long_tokens))
            raw_length.append(len(full_label) + 1 + len(name))
            formatted.append(full_label + ' ' + decorated_name)
        else:  # impossible if the align parameter has been sanitized.
            raise ValueError('Unrecognized alignment option %s' % align)
        return max(raw_length), formatted

    def MakeOptionLine(self, label, doc, label_width):
        decorator_length = self._decorator.GetDecoratorLength(label)
        format_string = '\n  %%-%ds%%s' % (label_width + decorator_length)
        return format_string % (label, doc)


class BaseHelpFormatter(object):
    """Base formatter that formats the help messages of a parser."""

    def __init__(self, component_maker=None, **kw):
        """Initializes the BaseHelpFormatter object.
        Args:
            component_maker: a strategy object to make various components in
                the help message.
            **kw: options like label alignment and others.
        """
        self._component_maker = component_maker
        if None == self._component_maker:
            self._component_maker = _PlainComponentMaker()
        self._SetOptions(**kw)

    def _SetOptions(self, **kw):
        self._align = _AlignmentOption.both
        for key, value in kw.iteritems():
            if 'align' == key:
                if hasattr(_AlignmentOption, value):
                    self._align = getattr(_AlignmentOption, value)
                else:
                    raise ValueError('unknown align parameter %s' % value)

    def Format(self, parser):
        """Formats relevant information of the parser into a help message."""
        output_strings = []
        output_strings.extend(self._component_maker.MakeHeader(parser))

        option_list = sorted(set(parser._option_map.values()),
                             key=lambda option: option.GetCliTokens()[0])
        output_data = []
        label_width_per_option = []
        # Calculate the width of the labels.
        for option in option_list:
            raw_width, formatted_labels = self._component_maker.MakeLabel(
                option, parser, self._align)
            label_width_per_option.append(raw_width)
            output_data.append([formatted_labels, option.doc,
                                option.GetDefaultValue()])
        label_width = max(label_width_per_option) + 3
        # Write out help messages of options.
        write = output_strings.append
        for item in output_data:
            tokens_with_names, doc, default_value = item[0], item[1], item[2]
            for i in range(len(tokens_with_names) - 1):
                write('\n  ' + tokens_with_names[i])
            message = self._component_maker.MakeOptionLine(
                tokens_with_names[-1], doc, label_width)
            logging.debug('default_value = %s for option %s', default_value, tokens_with_names)
            if default_value != None and default_value is not False:
                # Implementation: be ware that 0 == False, !=  ->  "is not" 
                message += ' (default: %s)' % default_value
            write(message)
        # Write out sub-commands
        if {} != parser._sub_command_map:
            write('\n\nCommands:')
            command_width = max([len(command) for command in
                                 parser._sub_command_map.keys()])
            format_string = '\n  %%-%ds  %%s' % (command_width + 2)
            for command, option in parser._sub_command_map.iteritems():
                write(format_string % (command, option.doc))
        return ''.join(output_strings)


class BashCompleteHelpFormatter(BaseHelpFormatter):
    """A help formatter that outputs bash complete script for a parser."""
    def __init__(self):
        pass

    def Format(self, parser):
        output = []
        write = output.append
        prog = os.path.basename(sys.argv[0])

        # Group options by their prefices.
        options_by_prefix = {}
        for token in parser._option_map.keys():
            prefix = token[0]
            if options_by_prefix.has_key(prefix):
                options_by_prefix[prefix].append(token)
            else:
                options_by_prefix[prefix] = [token]

        command_condition = ' && '.join(['${COMP_WORDS[i]} != %s*' % prefix
                                         for prefix in options_by_prefix.keys()])
        write(    """\n_%(prog)s()"""
                  """\n{"""
                  """\n    local cur cmd"""
                  """\n    COMPREPLY=()"""
                  """\n    cur="${COMP_WORDS[COMP_CWORD]}" """
                  """\n    for (( i=1; $i<$COMP_CWORD; i++ )); do"""
                  """\n        if [[ %(command_condition)s ]]; then"""
                  """\n            cmd="${COMP_WORDS[i]}" """
                  """\n            break"""
                  """\n        fi"""
                  """\n    done""" % vars())

        command_list = parser._sub_command_map.keys()
        ifelif = 'if'
        write(    """\n    if [[ -z $cmd ]]; then""")
        for prefix in options_by_prefix.keys():
            write("""\n        %s [[ ${cur} == %s* ]] ; then"""
                  """\n            COMPREPLY=( $(compgen -W "%s" -- ${cur}) )"""
                % (ifelif, prefix, ' '.join(options_by_prefix[prefix])))
            ifelif = 'elif'
        if [] != command_list:
            ifelse = 'if' if ifelif == 'if' else 'else'
            write("""\n        %s"""
                  """\n            COMPREPLY=( $(compgen -W "%s" -- ${cur}) )"""
                  % (ifelse, ' '.join(command_list)))
        write(    """\n        fi""" )
        for command in command_list:
            opts = ' '.join(
                parser._sub_command_map[command].parser._option_map.keys())
            write("""\n    elif [[ $cmd == "%(command)s" ]]; then"""
                  """\n        COMPREPLY=( $(compgen -W "%(opts)s" -- ${cur}) )"""
                  % vars())
        write(    """\n    fi"""
                  """\n    return 0"""
                  """\n}"""
                  """\ncomplete -F _%(prog)s %(prog)s""" % vars())
        return ''.join(output)


class CliParser(object):
    """A builder for command line interface."""
    _OPTION = 0
    _OPTION_AND_VALUE = 1
    _ARGUMENT = 2
    read_lock = threading.RLock()
    write_lock = threading.RLock()

    def __init__(self, description='', add_help=True,
                 short_option_prefix='-', short_option_length=1,
                 long_option_prefix='--', long_option_delimiter='='):
        if short_option_prefix.startswith(long_option_prefix):
            raise ValueError('The long_option_prefix can not be a '
                             'substring of short_option_prefix')
        self._description = description
        self._short_option_prefix = short_option_prefix
        self._short_option_length = short_option_length
        self._long_option_prefix = long_option_prefix
        self._long_option_delimiter = long_option_delimiter
        self._option_map = {}  # map of token name or alias to _CliOption
        self._named_arguments = []
        self._sub_command_map = {}
        self._program_name = ''
        self._delegates = []  # delegate parsers for unknown options
        self._option_to_build = None
        # Create the helper action
        if add_help:
            class HelpSetting(object):
                def __init__(self, arg):
                    logging.debug('HelpSetting(): arg = %s', arg)
                    splitted = arg.split('.')  # decoration.align
                    accepted = ('', 'plain', 'ansi', 'bash_complete')
                    if splitted[0] not in accepted:
                        raise ValueError('%s is not one of recognized values %s' %
                                         (splitted[0], accepted))
                    self.deco = splitted[0]
                    if '' == self.deco:
                        self.deco = 'plain'
                    self.align = _AlignmentOption.short
                    if len(splitted) > 1:
                        #value = 'ALIGN_' + splitted[1].upper()
                        value = splitted[1].lower()
                        if hasattr(_AlignmentOption, value):
                            self.align = getattr(_AlignmentOption, value)
                        else:
                            raise ValueError('%s is not one of recognized alignment' % splitted[1])
                def __repr__(self):
                    return '%s,%s' % (self.deco, self.align)
            (self.Take(self._long_option_prefix + 'help')
                .Alias(self._short_option_prefix + 'h')
                .As(HelpSetting).For('help_format').WithDefault(HelpSetting(''))
                .Will('Display this help message and exit. Help_format can be '
                      'plain, ansi or bash_complete.\n'
                      'The ansi option make help messages easier to read on ANSI terminals.\n'
                      'The bash_complete option makes scripts that you can source to enable bash\'s command '
                      'line completion for your script.\n'
                      'An optional alignment can be used with plain or ansi formats.\n'
                      'Try plain.both or ansi.short to see the effects.'))
            self._option_to_build.AddAction(self._PrintHelpAndExit)
            # Reset the option to build.
            self._option_to_build = None

    def Take(self, cli_token):
        """Sets up a new option to build.
        Args:
            cli_token: the CLI token for the new option to create.
        Returns the CliParser object.
        """
        self._option_to_build and self._option_to_build.AssertValid()
        if self._option_map.has_key(cli_token):  # overwrite not allowed
            raise ValueError('Overwriting options is not supported, '
                             're-appearing option %s' % cli_token)
        new_option = _CliOption(cli_token)
        self._option_to_build = self._option_map[cli_token] = new_option
        return self

    def Alias(self, alias):
        """Adds an alias to the option to build.
        Args:
            alias: an alias name for the option.
        Returns the CliParser object.
        """
        if None == self._option_to_build:
            raise CalledBeforeTakeError
        if self._option_map.has_key(alias):  # overwrite not allowed
            raise ValueError('Overwriting options is not supported, '
                             're-appearing option %s' % alias)
        self._option_to_build.AddAlias(alias)
        self._option_map[alias] = self._option_to_build
        return self

    def Aka(self, alias):
        """[Also known as], the same as Alias."""
        return self.Alias(alias)

    def AlsoKnownAs(self, alias):
        """Also known as, the same as Alias."""
        return self.Alias(alias)

    def As(self, value_type):
        """Sets the value type of the option to build.
        Args:
            value_type: the value type like bool, int, str, etc.
        Returns the CliParser object.
        """
        if None == self._option_to_build:
            raise CalledBeforeTakeError
        self._option_to_build.SetValueType(value_type)
        return self

    def Named(self, name):
        """Sets the name of the variable that the option operates on.
        Args:
            name: the name of the option, a string.
        Returns the CliParser object.
        """
        if None == self._option_to_build:
            raise CalledBeforeTakeError
        self._option_to_build.SetName(name)
        return self

    def For(self, name):
        return self.Named(name)

    def WithDefault(self, default_value):
        """Sets the default value of the option to build.
        Args:
            default_value: the default value.
        Returns the CliParser object.
        """
        if None == self._option_to_build:
            raise CalledBeforeTakeError
        self._option_to_build.SetDefaultValue(default_value)
        return self

    def Will(self, doc):
        """Sets the documentation of the option or argument, good for options that
        represent an action (a verb).
        Args:
            doc: a string.
        Returns the CliParser object.
        """
        if None == self._option_to_build:
            raise CalledBeforeTakeError
        self._option_to_build.doc = doc
        return self

    def Is(self, doc):
        """Syntactic sugar for Will(), good for options or arguments that represent
        a noun.
        Args:
            doc: a string.
        Returns the CliParser object.
        """
        return self.Will(doc)

    def Argument(self, multiplicity=1):
        """Marks the current token as an argument or arguments."""
        if None == self._option_to_build:
            raise CalledBeforeTakeError
        arg = self._option_to_build
        for cli_token in arg.GetCliTokens():
            del self._option_map[cli_token]
        arg._multiplicity = multiplicity
        if not arg.GetValueType():
            arg.SetValueType(str)
        self._named_arguments.append(arg)
        return self

    def Command(self, parser):
        """Marks the current token as a sub-command."""
        if None == self._option_to_build:
            raise CalledBeforeTakeError
        self._option_to_build.parser = parser
        self._option_to_build.SetValueType(type(parser))
        cli_token = self._option_to_build.GetCliTokens()[0]
        del self._option_map[cli_token]
        self._sub_command_map[cli_token] = self._option_to_build
        return self

    def MapBy(self, action):
        """Sets the mapping function of the option value.
        Args:
            action: a callable.
        Returns the CliParser object.
        """
        if None == self._option_to_build:
            raise CalledBeforeTakeError
        self._option_to_build.AddAction(
            lambda x: action(self._option_to_build.GetValueType()(x)))
        return self

    def Use(self, parser):
        """Use the given parser as a delegate for parsing unknown options.
        This method is guarded by a write lock.
        Args:
            parser: the parser object to use.
        Returns the CliParser object.
        Raises:
            TypeError if the parser is not of the right type.
            CyclicUseError if the parser already uses self and would result in
                a cyclic dependency.
        """
        if not hasattr(parser, 'Parse') or not callable(parser.Parse):
            raise TypeError('Can not use a parser without a Parse() method.')
        with self.write_lock:
            if parser._HasUsed(self):
                raise CyclicUseError('Can not use parser which already used self.')
            self._delegates.append(parser)
            return self

    def _HasUsed(self, parser):
        """Check if the given parser is used as a delegate parser.
        Args:
            parser: the parser object to check against delegate parsers.
        Returns:
            True if it is already used, False otherwise.
        """
        with self.read_lock:
            if [] == self._delegates:
                return False
            if parser in self._delegates:
                return True
            for delegate in self._delegates:
                if delegate._HasUsed(parser):
                    return True
            return False

    def Parse(self, argv):
        """Parse the argv list and builds a _CliConfig object.
        Args:
            argv: the list of command line arguments.
        Returns the constructed _CliConfig object.
        """
        self._option_to_build and self._option_to_build.AssertValid()
        self._program_name = argv[0]
        now, last, config = 1, len(argv) - 1, _CliConfig()
        self._FillDefaults(config)
        named_arguments = self._named_arguments[:]
        while now <= last:
            token_type, token, name, value = self._ParseToken(argv[now])
            logging.debug('self._ParseToken(%s) returns (%s,%s,%s,%s)',
                          argv[now], token_type, token, name, value)
            # fail fast
            if (token_type in [self._OPTION, self._OPTION_AND_VALUE] and
                not self._option_map.has_key(name)):
                self._DelegateParse(argv[now], config)
                now += 1
                continue
            if self._ARGUMENT == token_type:
                if self._sub_command_map.has_key(token):
                    sub_config = self._sub_command_map[token].parser.Parse(argv[now:])
                    config.SetOption(token, sub_config)
                    break
                else:
                    try:
                        arg = named_arguments.pop(0)
                        value = token
                        #print '%s arg = %s, %s value = %s' % (type(arg), arg,
                        #                                      type(value), value)
                        for action in arg.GetAction():
                            value = action(value)
                            #print '%s value = %s after action()' % (
                            #    type(value), value)
                    except:
                        value = token
                    config.AddArgument(value)
            elif self._OPTION == token_type:
                option = self._option_map[name]
                if bool == option.GetValueType():  # bool: toggles
                    self._SetOption(option, not option.GetDefaultValue(), config)
                else:
                    if now < last:
                        # Peek ahead
                        next_token_type, _, _, _ = self._ParseToken(argv[now + 1])
                        if next_token_type == self._ARGUMENT:
                            now += 1
                            self._SetOption(option, argv[now], config)
                        else:
                            self._SetOption(option, '', config)
                    else:
                        self._SetOption(option, '', config)
            elif self._OPTION_AND_VALUE == token_type:
                option = self._option_map[name]
                self._SetOption(option, value, config)
            else:  # impossible!
                raise AssertionError('Unknown token type!!! This is a serious '
                                     'bug. Notify the developer immediately!')
            now += 1
        self._AssignArguments(config)
        return config

    def _DelegateParse(self, token, config):
        """Delegates parsing work.
        Args:
            token: the CLI token.
            config: the config to set options to.
        Returns: None
        Raises: ValueError if the token is not recognized.
        """
        result = None
        for parser in self._delegates:
            try:
                result = parser.Parse(['', token])
                break
            except ValueError:
                pass
        if None == result:
            raise ValueError('unknown option name %s' % token)
        config.Merge(result)

    def _AssignArguments(self, config):
        """Assign positional arguments to named arguments."""
        if [] == self._named_arguments:
            return
        arg_index, arg_remaining = 0, len(config._arguments)
        for arg in self._named_arguments:
            mul, token = arg._multiplicity, arg.GetCliTokens()[0]
            if int == type(mul):
                if mul <= arg_remaining:
                    if mul == 1:  # don't make it a list
                        config.SetOption(token, config._arguments[arg_index])
                    else:
                        config.SetOption(
                            token, config._arguments[arg_index:arg_index+mul])
                    arg_index += mul
                    arg_remaining -= mul
                else:
                    if None != arg.GetDefaultValue():
                        values = config._arguments[arg_index:]
                        values.extend([arg.GetDefaultValue()] * (mul - len(values)))
                        config.SetOption(token, values)
                    else:
                        raise ValueError('Not enough arguments.')
            elif 0 == arg_remaining:
                if ONE_OR_MORE == mul:
                    raise ValueError('Missing arguments.')
                elif None != arg.GetDefaultValue():
                    config.SetOption(token, [arg.GetDefaultValue()])
                else:
                    config.SetOption(token, [])
            elif mul in (ONE_OR_MORE, ZERO_OR_MORE):
                config.SetOption(token, config._arguments[arg_index:])
                arg_index = len(config._arguments)
                arg_remaining = 0
            elif ZERO_OR_ONE == mul:
                config.SetOption(token, [config._arguments[arg_index]])
                arg_index += 1
                arg_remaining -= 1

    def _SetOption(self, option, value, cli_config):
        logging.debug('_SetOption(): action is %s, value = "%s"', option.GetAction(), value)
        for action in option.GetAction():  # transform the value in succession
            value = action(value)
        cli_config.SetOption(self._Canonicalize(option.GetCliTokens()[0]),
                             value)

    def _FillDefaults(self, cli_config):
        """Fills the config with default values of options.
        Arguments:
            cli_config: an object with SetOption() method, usually a
            _CliConfig object.
        """
        for cli_token, option in self._option_map.iteritems():
            default_value = option.GetDefaultValue()
            if None != default_value:
                cli_config.SetOption(self._Canonicalize(cli_token), default_value)

    def _Canonicalize(self, cli_token):
        """Returns the canonical form of an option's cli token.
        Arguments:
            cli_token: the cli token of the option.
        """
        if cli_token.startswith(self._long_option_prefix):
            return cli_token[len(self._long_option_prefix):]
        elif cli_token.startswith(self._short_option_prefix):
            return cli_token[len(self._short_option_prefix):]
        # impossible!
        logging.error('%s is not an option token, long prefix = %s, short prefix = %s' % (cli_token, self._long_option_prefix, self._short_option_prefix))
        raise ValueError('%s is not an option token' % cli_token)

    def _ParseToken(self, token):
        """Parse a token against option patterns to categorize it.
        Arguments:
            token: a token in the command line.
        Returns:
            A tuple of (token_type, token, name, value).
            The token_type is one of three enums:
                _OPTION:  if the token is an name with no value.
                _OPTION_AND_VALIUE:  if the token is a name plus a value.
                _ARGUMENT:  if the token is an argument.
            The token is the input token untouched.
            The name is the name of the option.
            The value is the value found in the token for _OPTION_AND_VALIUE
            type.
        """
        if token.startswith(self._long_option_prefix):  # long option
            equal = token.find(self._long_option_delimiter)
            if -1 == equal:
                return (self._OPTION, token, token, None)
            else:
                return (self._OPTION_AND_VALUE, token, token[:equal],
                        token[equal + len(self._long_option_delimiter):])
        elif token.startswith(self._short_option_prefix):  # short option
            end = len(self._short_option_prefix) + self._short_option_length
            if end == len(token):
                return (self._OPTION, token, token[:end], None)
            else:
                return (self._OPTION_AND_VALUE, token, token[:end], token[end:])
        else:
            return (self._ARGUMENT, token, None, None)

    def GetHelpMessage(self, formatter=None, align=_AlignmentOption.short):
        """Returns the help messages as a string.
        Args:
            formatter: The format of the help message, is one of 'terminal',
                'html' or 'plain' (the default). Only 'plain' is implemented now.
        Returns:
            The formatted help message as a string.
        """
        if formatter in ('plain', None):
            formatter = BaseHelpFormatter(_PlainComponentMaker(), align=align)
        elif 'ansi' == formatter:
            formatter = BaseHelpFormatter(_PlainComponentMaker(
                _AnsiTerminalDecorator()), align=align)
        elif 'bash_complete' == formatter:
            formatter = BashCompleteHelpFormatter()
        elif (not hasattr(formatter, 'Format') or not callable(formatter.Format)):
            # not a formatter
            raise TypeError('formatter must be a string or an object with a Format method.')
        return formatter.Format(self)

    def _PrintHelpAndExit(self, help_setting):
        logging.debug('_PrintHelpAndExit(): help_setting = %s', help_setting)
        print self.GetHelpMessage(help_setting.deco, align=help_setting.align)
        sys.exit(0)

